
#include "cl_base_Data.h"

void CDataSetForClassification::InitValidFeatures()
{
}

int CDataSetForClassification::GetValidFeatureAmount()
{
	return GetFeatureDimension();
}

bool CDataSetForClassification::IsFeatureValid(int FeatureIndex)
{
	return true;
}

void CDataSetForClassification::UpdateValidFeatures()
{
}

double CDataSetForClassification::GetDataDensity()
{
	return 1.0;
}

double CDataSetForClassification::ComputeDataDensity()
{
	double ret = 0.0;
	int ea = this->GetExampleAmount();
	int fd = this->GetFeatureDimension();
	CSparseVector tSparseVector;
	int sv = (int)(ea * 0.05); if (sv < 100) sv = 100;
	int iv;
	for ( iv = 0; iv < sv; ++iv )
	{
		this->MakeSparseVector_ByExample(tSparseVector, CMyRandom::NextInt(ea));
		ret += tSparseVector.Length;
	}
	return ret / fd / sv;
}

void CDataSetForClassification::MakeSparseVector_ByExample( CSparseVector& DstFeatureVector, int SrcExampleIndex )
{
	int i, ea, fd;
	double t;
	ea = this->GetExampleAmount();
	fd = this->GetFeatureDimension();
	DstFeatureVector.Clear();
	if (SrcExampleIndex<0 || SrcExampleIndex>=ea) return;
	for ( i = 0; i < fd; ++i )
	{
		t = this->GetExampleFeature(SrcExampleIndex, i);
		if (t<-MINIMUM_DOUBLE_EPS_C || t>+MINIMUM_DOUBLE_EPS_C)
			DstFeatureVector.PushBack(i, t);
	}
}

void CDataSetForClassification::MakeSparseVector_ByFeature( CSparseVector& DstFeatureVector, int SrcFeatureIndex )
{
	int i, ea, fd;
	double t;
	ea = this->GetExampleAmount();
	fd = this->GetFeatureDimension();
	DstFeatureVector.Clear();
	if (SrcFeatureIndex<0 || SrcFeatureIndex>=fd) return;
	for ( i = 0; i < ea; ++i )
	{
		t = this->GetExampleFeature(i, SrcFeatureIndex);
		if (t<-MINIMUM_DOUBLE_EPS_C || t>+MINIMUM_DOUBLE_EPS_C)
			DstFeatureVector.PushBack(i, t);
	}
}

// [*] CLabeledDataSetForClassification_Proxy

CLabeledDataSetForClassification_Proxy::CLabeledDataSetForClassification_Proxy()
{
	_pSource = NULL;
	_pProxy = NULL;
}

CLabeledDataSetForClassification_Proxy::~CLabeledDataSetForClassification_Proxy()
{
	ClearProxyMap();
}

void CLabeledDataSetForClassification_Proxy::InitSourceAndProxy(CLabeledDataSetForClassification* pSource, CLabeledDataSetForClassification* pProxy)
{
	_pSource = pSource;
	_pProxy = pProxy;
	ClearProxyMap();
}

void CLabeledDataSetForClassification_Proxy::ClearProxyMap()
{
	ProxyMap.clear();
	LastQuery = -1;
}

void CLabeledDataSetForClassification_Proxy::SetProxy(int ExampleIndexInSource, int ExampleIndexInProxy)
{
	if (_pSource==NULL || _pProxy==NULL) return;
	if (ExampleIndexInSource<0 || ExampleIndexInSource>=_pSource->GetExampleAmount()) return;
	if (ExampleIndexInProxy<0 || ExampleIndexInProxy>=_pProxy->GetExampleAmount()) return;
	ProxyMap[ExampleIndexInSource] = ExampleIndexInProxy;
	LastQuery = -1;
}

int CLabeledDataSetForClassification_Proxy::GetFeatureDimension()
{
	return (_pSource==NULL ? 0 : _pSource->GetFeatureDimension());
}

int CLabeledDataSetForClassification_Proxy::GetExampleAmount()
{
	return (_pSource==NULL ? 0 : _pSource->GetExampleAmount());
}

int CLabeledDataSetForClassification_Proxy::GetExampleLabel(int ExampleIndex)
{
	if (_pSource==NULL) return 0;
	if (ExampleIndex!=LastQuery || LastQuery<0)
	{
		itr = ProxyMap.find(ExampleIndex);
		LastQuery = ExampleIndex;
	}
	if (itr != ProxyMap.end())
		return _pProxy->GetExampleLabel(itr->second);
	else
		return _pSource->GetExampleLabel(ExampleIndex);
}

double CLabeledDataSetForClassification_Proxy::GetExampleInitialWeight(int ExampleIndex)
{
	if (_pSource==NULL) return 1.0;
	if (ExampleIndex!=LastQuery || LastQuery<0)
	{
		itr = ProxyMap.find(ExampleIndex);
		LastQuery = ExampleIndex;
	}
	if (itr != ProxyMap.end())
		return _pProxy->GetExampleInitialWeight(itr->second);
	else
		return _pSource->GetExampleInitialWeight(ExampleIndex);
}

double CLabeledDataSetForClassification_Proxy::GetExampleFeature(int ExampleIndex, int FeatureIndex)
{
	if (_pSource==NULL) return 0.0;
	if (ExampleIndex!=LastQuery || LastQuery<0)
	{
		itr = ProxyMap.find(ExampleIndex);
		LastQuery = ExampleIndex;
	}
	if (itr != ProxyMap.end())
		return _pProxy->GetExampleFeature(itr->second, FeatureIndex);
	else
		return _pSource->GetExampleFeature(ExampleIndex, FeatureIndex);
}

// [*] CLabeledDataSetForClassification_ByExample

void CLabeledDataSetForClassification_ByExample::GenerateFeatureFile(
	string strDstDataFileName,
	CLabeledDataSetForClassification* pDataSetByExample,
	size_t FEATURE_POOL_SIZE_ScaleMB
	)
{
	CWeightedClassificationExample twn;
	CWeightedClassificationExampleList ExampleList;
	ExampleList.Clear();
	int i;
	for ( i = 0; i < pDataSetByExample->GetExampleAmount(); ++i )
	{
		twn.ExampleIndex = i;
		twn.Weight = pDataSetByExample->GetExampleInitialWeight(i);
		ExampleList.PushBack(twn);
	}
	ExampleList.ComputeTotalExampleWeight();

	GenerateFeatureFile(strDstDataFileName, pDataSetByExample, ExampleList, false, FEATURE_POOL_SIZE_ScaleMB);
}

void CLabeledDataSetForClassification_ByExample::GenerateFeatureFile(
	string strDstDataFileName,
	CLabeledDataSetForClassification* pDataSetByExample,
	const CWeightedClassificationExampleList& ExampleList, bool fUseWeightInList,
	size_t FEATURE_POOL_SIZE_ScaleMB
	)
{
	int i,j,o,jj;
	int _na,fd;
	int* pLabel;
	double* pInitialWeight;
	double* pValue;
	double* pd;

	_na = ExampleList.ExampleAmount;
	CSimpleTypeArray<int> niList;
	niList.Clear();
	for ( i = 0; i < _na; ++i) 
		niList.PushBack(ExampleList.rgExample[i].ExampleIndex);
	const int* qni = niList.pElement;

	fd = pDataSetByExample->GetFeatureDimension();
	FILE* FOUT = fopen(strDstDataFileName.c_str(),"wb");
	pLabel = new int[_na];
	pInitialWeight = new double[_na];

	CDataStructureTools::my_fwrite_single(FOUT, _na);
	CDataStructureTools::my_fwrite_single(FOUT, fd);
	//CDataStructureTools::my_fwrite(FOUT, _na, sizeof(_na), 1);
	//CDataStructureTools::my_fwrite(FOUT, fd, sizeof(fd), 1);
	for ( i = 0; i < _na; ++i )
	{
		pLabel[i] = pDataSetByExample->GetExampleLabel(qni[i]);
		if (fUseWeightInList)
			pInitialWeight[i] = ExampleList.rgExample[i].Weight;
		else
			pInitialWeight[i] = pDataSetByExample->GetExampleInitialWeight(qni[i]);
	}
	CDataStructureTools::my_fwrite(FOUT, pLabel, _na);
	CDataStructureTools::my_fwrite(FOUT, pInitialWeight, _na);

	size_t _ui64_DataBlockSize = sizeof(double) * (size_t)fd;
	if (_ui64_DataBlockSize<1) _ui64_DataBlockSize = 1;
	int ExampleCacheAmount = (int)(FEATURE_POOL_SIZE_ScaleMB * (size_t)(1048576) / _ui64_DataBlockSize);
	if (ExampleCacheAmount>_na) ExampleCacheAmount = _na;
	pValue = new double [((size_t)fd) * (size_t)ExampleCacheAmount];

	for ( j = 0; j < _na; j += ExampleCacheAmount)
	{
		o = _na-j; if (o>ExampleCacheAmount) o = ExampleCacheAmount;
		for ( jj = 0; jj < o; ++jj )
		{
			pd = pValue + ((size_t)fd) * (size_t)jj;
			for ( i = 0; i < fd; ++i )
				*(pd++) = pDataSetByExample->GetExampleFeature(qni[j+jj], i);
			if (jj==0 || jj==o-1 || CMyRandom::NextInt(o/1000+1)==0)
			{
				CExternalTools::ConsoleTools::ClearCurrentLine();
				cout<<"Feature Generation Done: ("<<j+jj+1<<" / "<<j+o<<" / "<<_na<<"), ("<<fd<<");";
			}
		}
		cout<<endl<<"Feature Generation Done: writing data...";
		CDataStructureTools::my_fwrite(FOUT, pValue, ((size_t)fd) * (size_t)o);
	}
	CExternalTools::ConsoleTools::ClearCurrentLine();
	delete[] pInitialWeight;
	delete[] pLabel;
	delete[] pValue;
	fclose(FOUT);
}

void CLabeledDataSetForClassification_ByExample::myRelease()
{
	ExampleAmount = FeatureDim = 0;
	_ExamplePoolSize = 0;
	_FreeSpacePosition = -1;
	_BaseOffset = _DataBlockSize = 0;
	CDataStructureTools::MyRelease_List(rgExampleLabel);
	CDataStructureTools::MyRelease_List(pFeatureValuePool);
	CDataStructureTools::MyRelease_List(pExampleIndex);
	CDataStructureTools::MyRelease_List(pExampleHeader);
	if (FIN != NULL)
	{
		fclose(FIN);
		FIN = NULL;
	}
}

CLabeledDataSetForClassification_ByExample::CLabeledDataSetForClassification_ByExample()
{
	ExampleAmount = FeatureDim = 0;
	_ExamplePoolSize = 0;
	_FreeSpacePosition = -1;
	_BaseOffset = _DataBlockSize = 0;
	rgExampleLabel = NULL;
	pFeatureValuePool = NULL;
	pExampleIndex = NULL;
	pExampleHeader = NULL;
	FIN = NULL;
}

CLabeledDataSetForClassification_ByExample::~CLabeledDataSetForClassification_ByExample()
{
	myRelease();
}

void CLabeledDataSetForClassification_ByExample::myInit(string strSrcDataFileName, size_t FEATURE_POOL_SIZE_ScaleMB)
{
	myRelease();
	FIN = fopen(strSrcDataFileName.c_str(),"rb");
	CDataStructureTools::my_fread_single(FIN, ExampleAmount);
	CDataStructureTools::my_fread_single(FIN, FeatureDim);
	rgExampleLabel = new int [ExampleAmount];
	CDataStructureTools::my_fread(FIN, rgExampleLabel, ExampleAmount);
	InitialExampleWeightList.Resize(ExampleAmount);
	CDataStructureTools::my_fread(FIN, InitialExampleWeightList, ExampleAmount);
	_BaseOffset = sizeof(FeatureDim) + sizeof(ExampleAmount) + (sizeof(*rgExampleLabel) + sizeof(*InitialExampleWeightList.pElement)) * (size_t)ExampleAmount;
	_DataBlockSize = sizeof(*pFeatureValuePool) * (size_t)FeatureDim;
	_FEATURE_POOL_SIZE_ScaleMB = FEATURE_POOL_SIZE_ScaleMB;
	_ExamplePoolSize = (int)(_FEATURE_POOL_SIZE_ScaleMB * (size_t)(1048576) / _DataBlockSize);
	if (_ExamplePoolSize>ExampleAmount) _ExamplePoolSize = ExampleAmount;
	pFeatureValuePool = new double [((size_t)FeatureDim) * (size_t)_ExamplePoolSize];
	pExampleIndex = new int [ExampleAmount];
	memset(pExampleIndex, -1, sizeof(int) * (size_t)ExampleAmount);
	pExampleHeader = new double* [ExampleAmount];
	memset(pExampleHeader, NULL, sizeof(double*) * (size_t)ExampleAmount);
	_FreeSpacePosition = _ExamplePoolSize - 1;
}

int CLabeledDataSetForClassification_ByExample::GetFeatureDimension()
{
	return FeatureDim;
}

int CLabeledDataSetForClassification_ByExample::GetExampleAmount()
{
	return ExampleAmount;
}

int CLabeledDataSetForClassification_ByExample::GetExampleLabel(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount)
		return -1;
	else
		return rgExampleLabel[ExampleIndex];
}

double CLabeledDataSetForClassification_ByExample::GetExampleInitialWeight(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount)
		return 0.0;
	else
		return InitialExampleWeightList.pElement[ExampleIndex];
}

double CLabeledDataSetForClassification_ByExample::GetExampleFeature(int ExampleIndex, int FeatureIndex)
{
	int i;
	double ret = 0.0;

	if (pExampleHeader[ExampleIndex] == NULL)
	{
		if (_FreeSpacePosition>=0)
		{
			i = _FreeSpacePosition--;
		}
		else
		{
			i = CMyRandom::NextInt(_ExamplePoolSize);
			pExampleHeader[pExampleIndex[i]] = NULL;
		}
		pExampleIndex[i] = ExampleIndex;
		pExampleHeader[ExampleIndex] = pFeatureValuePool + ((size_t)FeatureDim) * (size_t)i;
		_fseeki64(FIN, _BaseOffset + _DataBlockSize * (size_t)ExampleIndex, SEEK_SET);
		CDataStructureTools::my_fread(FIN, pExampleHeader[ExampleIndex], FeatureDim);
	}
	ret = pExampleHeader[ExampleIndex][FeatureIndex];
	return ret;
}

// [*] CLabeledDataSetForClassification_ByFeature

void CLabeledDataSetForClassification_ByFeature::GenerateFeatureFile(
	string strDstDataFileName,
	CLabeledDataSetForClassification* pDataSetByExample,
	size_t FEATURE_POOL_SIZE_ScaleMB
	)
{
	CWeightedClassificationExample twn;
	CWeightedClassificationExampleList ExampleList;
	ExampleList.Clear();
	int i;
	for ( i = 0; i < pDataSetByExample->GetExampleAmount(); ++i )
	{
		twn.ExampleIndex = i;
		twn.Weight = pDataSetByExample->GetExampleInitialWeight(i);
		ExampleList.PushBack(twn);
	}
	ExampleList.ComputeTotalExampleWeight();

	GenerateFeatureFile(strDstDataFileName, pDataSetByExample, ExampleList, false, FEATURE_POOL_SIZE_ScaleMB);
}

void CLabeledDataSetForClassification_ByFeature::GenerateFeatureFile(
	string strDstDataFileName,
	CLabeledDataSetForClassification* pDataSetByExample,
	const CWeightedClassificationExampleList& ExampleList, bool fUseWeightInList,
	size_t FEATURE_POOL_SIZE_ScaleMB
	)
{
	int i,j,o,jj;
	int _na,fd;
	int* pLabel;
	double* pInitialWeight;
	double* pValue;
	double* pd;

	_na = ExampleList.ExampleAmount;
	CSimpleTypeArray<int> niList;
	niList.Clear();
	for ( i = 0; i < _na; ++i) 
		niList.PushBack(ExampleList.rgExample[i].ExampleIndex);
	const int* qni = niList.pElement;

	fd = pDataSetByExample->GetFeatureDimension();
	FILE* FOUT = fopen(strDstDataFileName.c_str(),"wb");
	pLabel = new int[_na];
	pInitialWeight = new double[_na];
	CDataStructureTools::my_fwrite_single(FOUT, fd);
	CDataStructureTools::my_fwrite_single(FOUT, _na);
	for ( i = 0; i < _na; ++i )
	{
		pLabel[i] = pDataSetByExample->GetExampleLabel(qni[i]);
		if (fUseWeightInList)
			pInitialWeight[i] = ExampleList.rgExample[i].Weight;
		else
			pInitialWeight[i] = pDataSetByExample->GetExampleInitialWeight(qni[i]);
	}
	CDataStructureTools::my_fwrite(FOUT, pLabel, _na);
	CDataStructureTools::my_fwrite(FOUT, pInitialWeight, _na);

	size_t _ui64_DataBlockSize = sizeof(double) * (size_t)_na;
	if (_ui64_DataBlockSize<1) _ui64_DataBlockSize = 1;
	int FeatureCacheAmount = (int)(FEATURE_POOL_SIZE_ScaleMB * (size_t)(1024) * (size_t)(1024) / _ui64_DataBlockSize);
	if (FeatureCacheAmount>fd) FeatureCacheAmount = fd;
	pValue = new double [((size_t)_na) * (size_t)FeatureCacheAmount];

	for ( j = 0; j < fd; j += FeatureCacheAmount)
	{
		o = fd-j; if (o>FeatureCacheAmount) o = FeatureCacheAmount;
		for ( i = 0; i < _na; ++i )
		{
			pd = pValue + i;
			for ( jj = 0; jj < o; ++jj )
			{
				*pd = pDataSetByExample->GetExampleFeature(qni[i], j+jj);
				pd += (size_t)_na;
			}
			if (i==_na-1 || CMyRandom::NextInt(_na/1000+1)==0)
			{
				CExternalTools::ConsoleTools::ClearCurrentLine();
				cout<<"Feature Generation Done: ("<<i+1<<" / "<<_na<<"), ("<<j+o<<" / "<<fd<<");";
			}
		}
		cout<<endl<<"Feature Generation Done: writing data...";
		CDataStructureTools::my_fwrite(FOUT, pValue, ((size_t)_na) * (size_t)o);
	}
	CExternalTools::ConsoleTools::ClearCurrentLine();
	delete[] pInitialWeight;
	delete[] pLabel;
	delete[] pValue;
	fclose(FOUT);
}

void CLabeledDataSetForClassification_ByFeature::myRelease()
{
	ExampleAmount = FeatureDim = 0;
	_FeaturePoolSize = 0;
	_FreeSpacePosition = -1;
	_BaseOffset = _DataBlockSize = 0;
	CDataStructureTools::MyRelease_List(rgExampleLabel);
	CDataStructureTools::MyRelease_List(pFeatureValuePool);
	CDataStructureTools::MyRelease_List(pFeatureIndex);
	CDataStructureTools::MyRelease_List(pFeatureHeader);
	if (FIN != NULL)
	{
		fclose(FIN);
		FIN = NULL;
	}
}

CLabeledDataSetForClassification_ByFeature::CLabeledDataSetForClassification_ByFeature()
{
	ExampleAmount = FeatureDim = 0;
	_FeaturePoolSize = 0;
	_FreeSpacePosition = -1;
	_BaseOffset = _DataBlockSize = 0;
	rgExampleLabel = NULL;
	pFeatureValuePool = NULL;
	pFeatureIndex = NULL;
	pFeatureHeader = NULL;
	FIN = NULL;
}

CLabeledDataSetForClassification_ByFeature::~CLabeledDataSetForClassification_ByFeature()
{
	myRelease();
}

void CLabeledDataSetForClassification_ByFeature::myInit(string strSrcDataFileName, size_t FEATURE_POOL_SIZE_ScaleMB)
{
	myRelease();
	FIN = fopen(strSrcDataFileName.c_str(),"rb");
	CDataStructureTools::my_fread_single(FIN, FeatureDim);
	CDataStructureTools::my_fread_single(FIN, ExampleAmount);
	rgExampleLabel = new int [ExampleAmount];
	CDataStructureTools::my_fread(FIN, rgExampleLabel, ExampleAmount);
	InitialExampleWeightList.Resize(ExampleAmount);
	CDataStructureTools::my_fread(FIN, InitialExampleWeightList, ExampleAmount);
	_BaseOffset = sizeof(FeatureDim) + sizeof(ExampleAmount) + (sizeof(*rgExampleLabel) + sizeof(*InitialExampleWeightList.pElement)) * (size_t)ExampleAmount;
	_DataBlockSize = sizeof(*pFeatureValuePool) * (size_t)ExampleAmount;
	_FEATURE_POOL_SIZE_ScaleMB = FEATURE_POOL_SIZE_ScaleMB;
	_FeaturePoolSize = (int)(_FEATURE_POOL_SIZE_ScaleMB * (size_t)(1024) * (size_t)(1024) / _DataBlockSize);
	if (_FeaturePoolSize>FeatureDim) _FeaturePoolSize = FeatureDim;
	pFeatureValuePool = new double [((size_t)ExampleAmount) * (size_t)_FeaturePoolSize];
	pFeatureIndex = new int [FeatureDim];
	memset(pFeatureIndex,-1, sizeof(int)*FeatureDim);
	pFeatureHeader = new double* [FeatureDim];
	memset(pFeatureHeader,0, sizeof(double*)*FeatureDim);
	_FreeSpacePosition = _FeaturePoolSize - 1;
}

int CLabeledDataSetForClassification_ByFeature::GetFeatureDimension()
{
	return FeatureDim;
}

int CLabeledDataSetForClassification_ByFeature::GetExampleAmount()
{
	return ExampleAmount;
}

int CLabeledDataSetForClassification_ByFeature::GetExampleLabel(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount)
		return -1;
	else
		return rgExampleLabel[ExampleIndex];
}

double CLabeledDataSetForClassification_ByFeature::GetExampleInitialWeight(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount)
		return 0.0;
	else
		return InitialExampleWeightList.pElement[ExampleIndex];
}

double CLabeledDataSetForClassification_ByFeature::GetExampleFeature(int ExampleIndex, int FeatureIndex)
{
	int i;
	double ret = 0.0;

	if (pFeatureHeader[FeatureIndex] == NULL)
	{
		if (_FreeSpacePosition>=0)
		{
			i = _FreeSpacePosition--;
		}
		else
		{
			i = CMyRandom::NextInt(_FeaturePoolSize);
			pFeatureHeader[pFeatureIndex[i]] = NULL;
		}
		pFeatureIndex[i] = FeatureIndex;
		pFeatureHeader[FeatureIndex] = pFeatureValuePool + ((size_t)ExampleAmount) * (size_t)i;
		_fseeki64(FIN, _BaseOffset + _DataBlockSize * (size_t)FeatureIndex, SEEK_SET);
		CDataStructureTools::my_fread(FIN, pFeatureHeader[FeatureIndex], ExampleAmount);
	}
	ret = pFeatureHeader[FeatureIndex][ExampleIndex];
	return ret;
}

// [*] C_LDS_Sparse_ByExample::CExampleHeader

// [*] C_LDS_Sparse_ByExample::CDataFileWriter

C_LDS_Sparse_ByExample::CDataFileWriter::CDataFileWriter()
{
	FOUT = NULL;
}

C_LDS_Sparse_ByExample::CDataFileWriter::~CDataFileWriter()
{
	Finish();
}

void C_LDS_Sparse_ByExample::CDataFileWriter::myInit(string strFN_DstData, int FeatureDimension)
{
	Finish();
	FOUT = fopen(strFN_DstData.c_str(), "wb");

	this->FeatureDimension = FeatureDimension;
	LastBlockOffset = sizeof(FeatureDimension);

	CDataStructureTools::my_fwrite_single(FOUT, FeatureDimension);
}

void C_LDS_Sparse_ByExample::CDataFileWriter::Finish()
{
	if (FOUT != NULL)
	{
		fclose(FOUT);
		FOUT = NULL;
		LastBlockOffset = 0;
	}
}

void C_LDS_Sparse_ByExample::CDataFileWriter::PushBack( const CSparseVector& FeatureVector, int Label, double InitialWeight )
{
	tExampleHeader.BlockOffset = FeatureVector.GetVectorMemorySize();
	tExampleHeader.Label = Label;
	tExampleHeader.InitialWeight = InitialWeight;
	CDataStructureTools::my_fwrite_single(FOUT, tExampleHeader);
	LastBlockOffset += sizeof(tExampleHeader);
	FeatureVector.WriteToFile(FOUT);
	LastBlockOffset += FeatureVector.GetVectorMemorySize();
}

void C_LDS_Sparse_ByExample::CDataFileWriter::ModifyFeatureDimension( int NewFeatureDimension )
{
	if (FOUT == NULL) return;

	this->FeatureDimension = NewFeatureDimension;
	__int64 pos = _ftelli64(FOUT);
	_fseeki64(FOUT, 0, SEEK_SET);
	CDataStructureTools::my_fwrite_single(FOUT, FeatureDimension);
	_fseeki64(FOUT, pos, SEEK_SET);
}

// [*] C_LDS_Sparse_ByExample

C_LDS_Sparse_ByExample::C_LDS_Sparse_ByExample()
{
	FIN = NULL;
	Clear();
}

C_LDS_Sparse_ByExample::~C_LDS_Sparse_ByExample()
{
	myRelease();
}

void C_LDS_Sparse_ByExample::Clear()
{
	ExampleAmount = 0; FeatureDimension = 0;
	IVPairCachePoolSize = 0; CachedIVPairAmount = 0;
	CachedExampleAmount = 0;
	HeaderList.DeepClear(); pExampleHeader = NULL;
	Example2CacheList.DeepClear(); pExample2Cache = NULL;
	Cache2ExampleList.DeepClear(); pCache2Example = NULL;
	CachedDataList.DeepClear(); pCachedData = NULL;
}

void C_LDS_Sparse_ByExample::myRelease()
{
	if (FIN != NULL)
	{
		fclose(FIN);
		FIN = NULL;
	}
}

void C_LDS_Sparse_ByExample::myInit( string strFN_Data, size_t FEATURE_POOL_SIZE_ScaleMB )
{
	myRelease();

	this->Clear();

	FIN = fopen(strFN_Data.c_str(), "rb");
	if (FIN == NULL) return;

	size_t LastOffset;
	size_t tBlockSize;
	CExampleHeader tExampleHeader;

	LastOffset = 0;
	CDataStructureTools::my_fread_single(FIN, FeatureDimension); LastOffset += sizeof(FeatureDimension);
	for ( ; ; )
	{
		if (!CDataStructureTools::my_fread_single(FIN, tExampleHeader)) break;
		LastOffset += sizeof(tExampleHeader);
		tBlockSize = tExampleHeader.BlockOffset;
		tExampleHeader.BlockOffset = LastOffset;
		LastOffset += tBlockSize;
		++ExampleAmount;
		HeaderList.PushBack(tExampleHeader);
		_fseeki64(FIN, tBlockSize, SEEK_CUR);
		//if (ExampleAmount%2000==0 || ExampleAmount>16780)
		//	ExampleAmount = ExampleAmount;
	}

	IVPairCachePoolSize = (size_t)(1048576.0 * FEATURE_POOL_SIZE_ScaleMB / CSparseVector::GetIVPairMemorySize() - ExampleAmount);
	if (IVPairCachePoolSize < (size_t)FeatureDimension) IVPairCachePoolSize = (size_t)FeatureDimension;

	pExampleHeader = HeaderList.pElement;
	CachedExampleAmount = 0;
	CachedDataList.Resize(ExampleAmount); CachedDataList.Clear(); pCachedData = CachedDataList.pElement;
	Cache2ExampleList.Resize(ExampleAmount); Cache2ExampleList.Clear(); pCache2Example = Cache2ExampleList.pElement;
	Example2CacheList.Resize(ExampleAmount); Example2CacheList.Fill(-1); pExample2Cache = Example2CacheList.pElement;
}

void C_LDS_Sparse_ByExample::RemoveCachedExample( int CachedIndex )
{
	if (CachedIndex<0 || CachedIndex>=CachedExampleAmount) return;
	int li = (int)CachedExampleAmount - 1;
	int xa = pCache2Example[CachedIndex];
	int ya = pCache2Example[li];
	pExample2Cache[ya] = CachedIndex;
	pCache2Example[CachedIndex] = ya;
	pCache2Example[li] = -1;
	pExample2Cache[xa] = -1;
	pCachedData[li].Swap(pCachedData[CachedIndex]);
	--CachedExampleAmount;
	CachedIVPairAmount -= pCachedData[li].Length;
	pCachedData[li].DeepClear();
	Cache2ExampleList.Resize(li);
	CachedDataList.Resize(li);
	
}

void C_LDS_Sparse_ByExample::NewCachedExample( int ExampleIndex )
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return;
	if (pExample2Cache[ExampleIndex] >= 0) return;
	pExample2Cache[ExampleIndex] = CachedExampleAmount;
	Cache2ExampleList.PushBack(ExampleIndex);
	CachedDataList.Resize(CachedExampleAmount + 1);
	CSparseVector& tSparseVector = pCachedData[CachedExampleAmount];
	const CExampleHeader& tExampleHeader = pExampleHeader[ExampleIndex];
	_fseeki64(FIN, tExampleHeader.BlockOffset, SEEK_SET);
	tSparseVector.ReadFromFile(FIN);
	CachedIVPairAmount += tSparseVector.Length;
	++CachedExampleAmount;
}

int C_LDS_Sparse_ByExample::GetExampleCacheIndex( int ExampleIndex )
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return -1;

	int RemovingCacheIndex;
	RemovingCacheIndex = pExample2Cache[ExampleIndex];
	if (RemovingCacheIndex >= 0) return RemovingCacheIndex;

	NewCachedExample(ExampleIndex);
	while (CachedIVPairAmount>IVPairCachePoolSize && CachedExampleAmount>1)
	{
		RemovingCacheIndex = CMyRandom::NextInt(CachedExampleAmount);
		if (RemovingCacheIndex == pExample2Cache[ExampleIndex]) continue;
		RemoveCachedExample(RemovingCacheIndex);
	}
	return pExample2Cache[ExampleIndex];
}

const CSparseVector& C_LDS_Sparse_ByExample::GetExampleFeatureSparseVector( int ExampleIndex )
{
	return pCachedData[this->GetExampleCacheIndex(ExampleIndex)];
}

int C_LDS_Sparse_ByExample::GetExampleAmount()
{
	return ExampleAmount;
}

double C_LDS_Sparse_ByExample::GetExampleInitialWeight( int ExampleIndex )
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return 1.0;
	return pExampleHeader[ExampleIndex].InitialWeight;
}

int C_LDS_Sparse_ByExample::GetFeatureDimension()
{
	return FeatureDimension;
}

double C_LDS_Sparse_ByExample::GetExampleFeature( int ExampleIndex, int FeatureIndex )
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return 0.0;
	int CacheIndex = GetExampleCacheIndex(ExampleIndex);
	return pCachedData[CacheIndex].GetValue(FeatureIndex);
}

int C_LDS_Sparse_ByExample::GetExampleLabel( int ExampleIndex )
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return 0;
	return pExampleHeader[ExampleIndex].Label;
}

double C_LDS_Sparse_ByExample::GetDataDensity()
{
	return this->ComputeDataDensity();
}

void C_LDS_Sparse_ByExample::MakeSparseVector_ByExample( CSparseVector& DstFeatureVector, int SrcExampleIndex )
{
	if (SrcExampleIndex<0 || SrcExampleIndex>=ExampleAmount)
	{
		DstFeatureVector.Clear();
		return;
	}
	int CacheIndex = GetExampleCacheIndex(SrcExampleIndex);
	DstFeatureVector.myInit(pCachedData[CacheIndex]);
}

//TODO

// CLabeledDataSetForClassification_InMemory

void CLabeledDataSetForClassification_InMemory::myInit_ByFeature(string strFN_DataByFeature, size_t FEATURE_POOL_SIZE_ScaleMB)
{
	CLabeledDataSetForClassification_ByFeature DataSetByFeature;
	DataSetByFeature.myInit(strFN_DataByFeature, FEATURE_POOL_SIZE_ScaleMB);
	myInit_ByFeature(DataSetByFeature);
}

void CLabeledDataSetForClassification_InMemory::myInit_ByFeature(CLabeledDataSetForClassification_ByFeature& DataSetByFeature)
{
	int i,j;
	double* pd;

	ExampleAmount = DataSetByFeature.GetExampleAmount();
	FeatureDimension = DataSetByFeature.GetFeatureDimension();
	FeatureList.Resize((size_t)ExampleAmount * (size_t)FeatureDimension);
	LabelList.Resize((size_t)ExampleAmount);
	WeightList.Resize((size_t)ExampleAmount);;
	for ( i = 0; i < ExampleAmount; ++i )
	{
		LabelList.pElement[i] = DataSetByFeature.GetExampleLabel(i);
		WeightList.pElement[i] = DataSetByFeature.GetExampleInitialWeight(i);
	}
	for ( j = 0; j < FeatureDimension; ++j )
	{
		pd = FeatureList.pElement + j;
		for ( i = 0; i < ExampleAmount; ++i )
		{
			*pd = DataSetByFeature.GetExampleFeature(i,j);
			pd += FeatureDimension;
		}
	}
}

//void CLabeledDataSetForClassification_InMemory::GenerateDataFile_ByExample(CLabeledDataSetForClassification* pDataSet, string strFN_DstData)
//{
//	cout<<"Under Construction..."<<endl;
//}
//
//void CLabeledDataSetForClassification_InMemory::myInit_ByExample(string strFN_Data)
//{
//	cout<<"Under Construction..."<<endl;
//}

void CLabeledDataSetForClassification_InMemory::myInit_ByExample(CLabeledDataSetForClassification* pDataSet)
{
	int i,j;
	double* pd;
	char strTemp[10001];

	ExampleAmount = pDataSet->GetExampleAmount();
	FeatureDimension = pDataSet->GetFeatureDimension();
	FeatureList.Resize((size_t)ExampleAmount * (size_t)FeatureDimension);
	LabelList.Resize((size_t)ExampleAmount);
	WeightList.Resize((size_t)ExampleAmount);;
	for ( i = 0; i < ExampleAmount; ++i )
	{
		LabelList.pElement[i] = pDataSet->GetExampleLabel(i);
		WeightList.pElement[i] = pDataSet->GetExampleInitialWeight(i);
	}
	pd = FeatureList.pElement;
	for ( i = 0; i < ExampleAmount; ++i )
	{
		for ( j = 0; j < FeatureDimension; ++j )
			*(pd++) = pDataSet->GetExampleFeature(i,j);
		sprintf_s(strTemp, "Feature Generation Done: <%d/%d>, <%d>", i+1, ExampleAmount, FeatureDimension);
		CExternalTools::ConsoleTools::ClearCurrentLine();
		cout<<strTemp;
	}
	cout<<endl;
}

void CLabeledDataSetForClassification_InMemory::ClearAndInit(int FeatureDim)
{
	this->FeatureDimension = FeatureDim;
	ExampleAmount = 0;
	FeatureList.Clear();
	LabelList.Clear();
	WeightList.Clear();
}

void CLabeledDataSetForClassification_InMemory::PushBack(int Label, const double* Feature, double InitialWeight)
{
	int j;
	size_t j_base;
	double* pd;

	++ExampleAmount;

	LabelList.PushBack(Label);
	j_base = FeatureList.Size;
	FeatureList.Resize(j_base + (size_t)FeatureDimension);
	pd = FeatureList.pElement + j_base;
	for ( j = 0; j < FeatureDimension; ++j )
		*(pd++) = *(Feature++);
	WeightList.PushBack(InitialWeight);
}

int CLabeledDataSetForClassification_InMemory::GetFeatureDimension()
{
	return FeatureDimension;
}

int CLabeledDataSetForClassification_InMemory::GetExampleAmount()
{
	return ExampleAmount;
}

int CLabeledDataSetForClassification_InMemory::GetExampleLabel(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return -1;
	return LabelList.pElement[ExampleIndex];
}

double CLabeledDataSetForClassification_InMemory::GetExampleInitialWeight(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return 0.0;
	return WeightList.pElement[ExampleIndex];
}

double CLabeledDataSetForClassification_InMemory::GetExampleFeature(int ExampleIndex, int FeatureIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return 0.0;
	if (FeatureIndex<0 || FeatureIndex>=FeatureDimension) return 0.0;
	return FeatureList.pElement[(size_t)FeatureIndex + (size_t)ExampleIndex * (size_t)FeatureDimension];
}

// [*] CWeightedClassificationExample

CWeightedClassificationExample::CWeightedClassificationExample()
{
	this->ExampleIndex = 0;
	this->Weight = 1.0;
}

CWeightedClassificationExample::CWeightedClassificationExample( int ExampleIndex, double Weight )
{
	this->ExampleIndex = ExampleIndex;
	this->Weight = Weight;
}

bool CWeightedClassificationExample::CompareByExampleIndex( const CWeightedClassificationExample& a, const CWeightedClassificationExample& b )
{
	return a.ExampleIndex < b.ExampleIndex;
}

bool CWeightedClassificationExample::CompareByWeight(const CWeightedClassificationExample& a, const CWeightedClassificationExample& b)
{
	return a.Weight < b.Weight;
}

// [*] CWeightedClassificationExampleList

CWeightedClassificationExampleList::CWeightedClassificationExampleList()
{
	Clear();
}

void CWeightedClassificationExampleList::myInit( CDataSetForClassification* pDataSet, bool fUseUniformedWeight )
{
	this->Clear();
	int i, ii;
	ii = pDataSet->GetExampleAmount();
	for ( i = 0; i < ii; ++i )
		this->PushBack(i, (fUseUniformedWeight ? 1.0 : pDataSet->GetExampleInitialWeight(i)));
}

void CWeightedClassificationExampleList::myInit( int ExampleAmount )
{
	this->Clear();
	int i;
	for ( i = 0; i < ExampleAmount; ++i )
		this->PushBack(i, 1.0);
}

void CWeightedClassificationExampleList::myInit( const CWeightedClassificationExampleList& SourceExampleList, double SampleRate )
{
	int i, r;
	rgExample.resize(SourceExampleList.ExampleAmount);
	r = 0;
	for ( i = 0; i < SourceExampleList.ExampleAmount; ++i )
	{
		if (CMyRandom::NextDouble() < SampleRate)
			rgExample[r++] = SourceExampleList.rgExample[i];
	}
	ExampleAmount = r;
	rgExample.resize(r);
	this->ComputeTotalExampleWeight();
}

void CWeightedClassificationExampleList::myInit( const CWeightedClassificationExampleList& SourceExampleList, int TargetExampleAmount )
{
	int i, r;
	rgExample.resize(SourceExampleList.ExampleAmount);
	r = 0;
	for ( i = 0; i < SourceExampleList.ExampleAmount; ++i )
	{
		if (CMyRandom::NextInt(SourceExampleList.ExampleAmount - i) < (TargetExampleAmount - r))
			rgExample[r++] = SourceExampleList.rgExample[i];
	}
	ExampleAmount = r;
	rgExample.resize(r);
	this->ComputeTotalExampleWeight();
}

void CWeightedClassificationExampleList::SortByIndex()
{
	sort(rgExample.begin(), rgExample.end(), CWeightedClassificationExample::CompareByExampleIndex);
}

void CWeightedClassificationExampleList::SortByWeight()
{
	sort(rgExample.begin(), rgExample.end(), CWeightedClassificationExample::CompareByWeight);
}

void CWeightedClassificationExampleList::RandomSplit( CSimpleTypeArray<CWeightedClassificationExampleList>& DstSubExampleList ) const
{
	if (DstSubExampleList.Size < 1) return;
	RandomSplit(DstSubExampleList, (int)DstSubExampleList.Size);
}

void CWeightedClassificationExampleList::RandomSplit( CSimpleTypeArray<CWeightedClassificationExampleList>& DstSubExampleList, int SubListAmount ) const
{
	DstSubExampleList.Resize(SubListAmount);
	int i, j;
	for ( j = 0; j < SubListAmount; ++j )
		DstSubExampleList.pElement[j].Clear();
	for ( i = 0; i < ExampleAmount; ++i )
	{
		j = CMyRandom::NextInt(SubListAmount);
		DstSubExampleList.pElement[j].PushBack(rgExample[i]);
	}
}

void CWeightedClassificationExampleList::RandomSplit(
	CWeightedClassificationExampleList& DstEL_A,
	CWeightedClassificationExampleList& DstEL_B,
	double EL_A_Ratio)
{
	DstEL_A.Clear(); DstEL_B.Clear();
	int i;
	for ( i = 0; i < ExampleAmount; ++i )
	{
		(CMyRandom::NextDouble()<EL_A_Ratio ? DstEL_A : DstEL_B).PushBack(rgExample[i]);
	}
}

void CWeightedClassificationExampleList::RandomSplit(
	CWeightedClassificationExampleList& DstEL_A,
	CWeightedClassificationExampleList& DstEL_B,
	int EL_A_Size)
{
	DstEL_A.Clear(); DstEL_B.Clear();
	int i, t;
	t = EL_A_Size;
	for ( i = 0; i < ExampleAmount; ++i )
	{
		if (CMyRandom::NextInt(ExampleAmount - i) < t)
		{
			DstEL_A.PushBack(rgExample[i]);
			--t;
		}
		else
			DstEL_B.PushBack(rgExample[i]);
	}
}

void CWeightedClassificationExampleList::AddEquals( const CWeightedClassificationExampleList& Another )
{
	int ii = Another.ExampleAmount;
	int i;
	for ( i = 0; i < ii; ++i )
		this->PushBack(Another.rgExample[i]);
}

void CWeightedClassificationExampleList::Clear()
{
	ExampleAmount = 0;
	TotalExampleWeight = 0.0;
	rgExample.clear();
}

void CWeightedClassificationExampleList::PushBack(const CWeightedClassificationExample& Example)
{
	++ExampleAmount;
	TotalExampleWeight += Example.Weight;
	rgExample.push_back(Example);
}

void CWeightedClassificationExampleList::PushBack( int ExampleIndex, double Weight )
{
	this->PushBack(CWeightedClassificationExample(ExampleIndex, Weight));
}

void CWeightedClassificationExampleList::ComputeTotalExampleWeight()
{
	vector<CWeightedClassificationExample>::iterator itr = rgExample.begin();
	vector<CWeightedClassificationExample>::iterator ied = rgExample.end();
	TotalExampleWeight = 0.0;
	for ( ; itr != ied; ++itr )
		TotalExampleWeight += itr->Weight;
}

void CWeightedClassificationExampleList::InitializeExampleWeightFromDataSet(CDataSetForClassification* pDataSet)
{
	vector<CWeightedClassificationExample>::iterator itr = rgExample.begin();
	vector<CWeightedClassificationExample>::iterator ied = rgExample.end();
	TotalExampleWeight = 0.0;
	for ( ; itr != ied; ++itr )
	{
		itr->Weight = pDataSet->GetExampleInitialWeight(itr->ExampleIndex);
		TotalExampleWeight += itr->Weight;
	}
}

void CWeightedClassificationExampleList::MultiplyEachWeightBy( double Ratio )
{
	int i;
	for ( i = 0; i < ExampleAmount; ++i )
		rgExample[i].Weight *= Ratio;
	TotalExampleWeight *= Ratio;
}

void CWeightedClassificationExampleList::NormalizeWeight( double TargetTotalExampleWeight )
{
	this->ComputeTotalExampleWeight();
	if ( TotalExampleWeight < MINIMUM_DOUBLE_EPS_C ) return;
	this->MultiplyEachWeightBy( TargetTotalExampleWeight / TotalExampleWeight );
}

// [*] CLabeledDataSetForClassification_ByGroupedFeature

void CLabeledDataSetForClassification_ByGroupedFeature::GenerateRandomFeatureIndex(CSimpleTypeArray<int>& DstRFI, CLabeledDataSetForClassification* pDataSet)
{
	int fd = pDataSet->GetFeatureDimension();
	int j, jj, o;
	DstRFI.Clear();
	for ( j = 0; j < fd; ++j )
		DstRFI.PushBack(j);
	for ( j = 0; j < fd; ++j )
	{
		jj = j + CMyRandom::NextInt(fd-j);
		o = DstRFI.pElement[j]; DstRFI.pElement[j] = DstRFI.pElement[jj]; DstRFI.pElement[jj] = o;
	}
}

void CLabeledDataSetForClassification_ByGroupedFeature::GenerateFeatureFile(
	string strDstDataFileName,
	CLabeledDataSetForClassification* pDataSetByExample,
	const CSimpleTypeArray<int>& RandomFeatureIndex,
	size_t FEATURE_POOL_SIZE_ScaleMB
	)
{
	CWeightedClassificationExample twn;
	CWeightedClassificationExampleList ExampleList;
	ExampleList.Clear();
	int i;
	for ( i = 0; i < pDataSetByExample->GetExampleAmount(); ++i )
	{
		twn.ExampleIndex = i;
		twn.Weight = pDataSetByExample->GetExampleInitialWeight(i);
		ExampleList.PushBack(twn);
	}
	ExampleList.ComputeTotalExampleWeight();

	GenerateFeatureFile(strDstDataFileName, pDataSetByExample, RandomFeatureIndex, ExampleList, false, FEATURE_POOL_SIZE_ScaleMB);
}

void CLabeledDataSetForClassification_ByGroupedFeature::GenerateFeatureFile(
	string strDstDataFileName,
	CLabeledDataSetForClassification* pDataSetByExample,
	const CSimpleTypeArray<int>& RandomFeatureIndex,
	const CWeightedClassificationExampleList& ExampleList, bool fUseWeightInList,
	size_t FEATURE_POOL_SIZE_ScaleMB
	)
{
	int i,j,o,jj;
	int _na;
	int fd;
	int* pLabel;
	//int* pFeatureIndex;
	const int* qFeatureIndex = RandomFeatureIndex.pElement;
	double* pValue;
	double* pInitialWeight;
	double* pd;

	_na = ExampleList.ExampleAmount;
	CSimpleTypeArray<int> niList;
	niList.Clear();
	for ( i = 0; i < _na; ++i) 
		niList.PushBack(ExampleList.rgExample[i].ExampleIndex);
	const int* qni = niList.pElement;

	fd = pDataSetByExample->GetFeatureDimension();
	if (fd != (int)RandomFeatureIndex.Size) return;

	FILE* FOUT = fopen(strDstDataFileName.c_str(),"wb");
	pLabel = new int[_na];
	pInitialWeight = new double[_na];
	CDataStructureTools::my_fwrite_single(FOUT, fd);
	CDataStructureTools::my_fwrite_single(FOUT, _na);
	for ( i = 0; i < _na; ++i )
	{
		pLabel[i] = pDataSetByExample->GetExampleLabel(qni[i]);
		if (fUseWeightInList)
			pInitialWeight[i] = ExampleList.rgExample[i].Weight;
		else
			pInitialWeight[i] = pDataSetByExample->GetExampleInitialWeight(qni[i]);
	}
	CDataStructureTools::my_fwrite(FOUT, pLabel, _na);
	CDataStructureTools::my_fwrite(FOUT, pInitialWeight, _na);

	//pFeatureIndex = new int[fd];
	//for ( j = 0; j < fd; ++j )
	//	pFeatureIndex[j] = j;
	//for ( j = 0; j < fd; ++j )
	//{
	//	jj = j + CMyRandom::NextInt(fd-j);
	//	o = pFeatureIndex[j]; pFeatureIndex[j] = pFeatureIndex[jj]; pFeatureIndex[jj] = o;
	//}
	//CDataStructureTools::my_fwrite(FOUT, pFeatureIndex, fd);
	CDataStructureTools::my_fwrite(FOUT, qFeatureIndex, fd);

	size_t _DataBlockSize = sizeof(double) * (size_t)_na;
	if (_DataBlockSize<1) _DataBlockSize = 1;
	int FeatureCacheAmount = (int)((size_t)FEATURE_POOL_SIZE_ScaleMB * (size_t)(1048576) / _DataBlockSize);
	if (FeatureCacheAmount>fd) FeatureCacheAmount = fd;
	pValue = new double [((size_t)_na) * (size_t)FeatureCacheAmount];

	CExternalTools::ConsoleTools::ClearCurrentLine();
	cout<<"Feature Generation Done: ("<<0<<" / "<<_na<<"), ("<<0<<" / "<<fd<<");";
	for ( j = 0; j < fd; j += FeatureCacheAmount)
	{
		o = fd-j; if (o>FeatureCacheAmount) o = FeatureCacheAmount;
		for ( i = 0; i < _na; ++i )
		{
			pd = pValue + i;
			for ( jj = 0; jj < o; ++jj )
			{
				*pd = pDataSetByExample->GetExampleFeature(qni[i],qFeatureIndex[j+jj]);
				pd += (size_t)_na;
			}
			if (i==_na-1 || CMyRandom::NextInt(_na/1000+1)==0)
			{
				CExternalTools::ConsoleTools::ClearCurrentLine();
				cout<<"Feature Generation Done: ("<<i+1<<" / "<<_na<<"), ("<<j+o<<" / "<<fd<<");";
			}
		}
		cout<<endl<<"Feature Generation Done: writing data...";
		CDataStructureTools::my_fwrite(FOUT, pValue, ((size_t)_na) * (size_t)o);
	}
	CExternalTools::ConsoleTools::ClearCurrentLine();
	//cout<<"Feature Generation Finished."<<endl;
	//delete[] pFeatureIndex;
	delete[] pInitialWeight;
	delete[] pLabel;
	delete[] pValue;
	fclose(FOUT);
}

void CLabeledDataSetForClassification_ByGroupedFeature::myRelease()
{
	if (FIN != NULL)
	{
		fclose(FIN);
		FIN = NULL;
	}

	ExampleAmount = FeatureDim = 0;
	LabelList.Clear();
	InitialWeightList.Clear();

	fea_index.Clear();
	fv_headers.Clear();
	fea2grp.Clear();

	fv_data.Clear();

	grp_amount = 0;
	per_grp_size = 0;
	grp_pool_size = 0;
	in_cache_fea_amount = 0;
	grp_headers.Clear();
	grp_length.Clear();
	grp2cache.Clear();
	cache2grp.Clear();
}

CLabeledDataSetForClassification_ByGroupedFeature::CLabeledDataSetForClassification_ByGroupedFeature()
{
	FIN = NULL;
	myRelease();
}

CLabeledDataSetForClassification_ByGroupedFeature::~CLabeledDataSetForClassification_ByGroupedFeature()
{
	myRelease();
}

void CLabeledDataSetForClassification_ByGroupedFeature::myInit(string strSrcDataFileName, int GroupAmount, size_t FEATURE_POOL_SIZE_ScaleMB)
{
	int i,j,o,tt;

	myRelease();
	FIN = fopen(strSrcDataFileName.c_str(),"rb");
	CDataStructureTools::my_fread_single(FIN, FeatureDim);
	CDataStructureTools::my_fread_single(FIN, ExampleAmount);

	LabelList.Resize(ExampleAmount);
	InitialWeightList.Resize(ExampleAmount);

	fea_index.Resize(FeatureDim);
	fv_headers.Resize(FeatureDim); fv_headers.Fill(NULL);
	fea2grp.Resize(FeatureDim); fea2grp.Fill(-1);

	CDataStructureTools::my_fread(FIN, LabelList, ExampleAmount);
	CDataStructureTools::my_fread(FIN, InitialWeightList, ExampleAmount);
	CDataStructureTools::my_fread(FIN, fea_index, FeatureDim);

	_BaseOffset = sizeof(FeatureDim) + sizeof(ExampleAmount) + 
		(size_t)ExampleAmount * (sizeof(*LabelList.pElement) + sizeof(*InitialWeightList.pElement)) + 
		(size_t)FeatureDim * sizeof(*fea_index.pElement);
	_DataBlockSize = sizeof(*fv_data.pElement) * (size_t)ExampleAmount;
	int _FeaturePoolSize = (int)((size_t)FEATURE_POOL_SIZE_ScaleMB * (size_t)1048576 / _DataBlockSize);
	//if (_FeaturePoolSize > FeatureDim) _FeaturePoolSize = FeatureDim;
	
	grp_amount = GroupAmount;
	if (grp_amount < 1) grp_amount = 1;
	per_grp_size = (FeatureDim + grp_amount - 1) / grp_amount;
	if (per_grp_size > _FeaturePoolSize) per_grp_size = _FeaturePoolSize;
	grp_amount = (FeatureDim + per_grp_size - 1) / per_grp_size;

	grp_pool_size = _FeaturePoolSize / per_grp_size;
	if (grp_pool_size > grp_amount)
		grp_pool_size = grp_amount;
	_FeaturePoolSize = per_grp_size * grp_pool_size;

	fv_data.Resize((size_t)ExampleAmount * (size_t)_FeaturePoolSize);

	grp_headers.Resize(grp_amount);
	grp_length.Resize(grp_amount);
	grp2cache.Resize(grp_amount); grp2cache.Fill(-1);
	cache2grp.Resize(grp_pool_size); cache2grp.Fill(-1);

	in_cache_fea_amount = 0;

	j = FeatureDim / grp_amount;
	o = FeatureDim % grp_amount;
	for ( i = 0; i < grp_amount; ++i )
		grp_length.pElement[i] = j + (i<o ? 1 : 0);
	o = 0;
	for ( i = 0; i < grp_amount; ++i )
	{
		grp_headers.pElement[i] = o;
		for ( j = 0; j < grp_length.pElement[i]; ++j )
		{
			tt = fea_index.pElement[o++];
			fea2grp.pElement[tt] = i;
		}
	}
}

int CLabeledDataSetForClassification_ByGroupedFeature::GetFeatureDimension()
{
	return FeatureDim;
}

int CLabeledDataSetForClassification_ByGroupedFeature::GetExampleAmount()
{
	return ExampleAmount;
}

int CLabeledDataSetForClassification_ByGroupedFeature::GetExampleLabel(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return -1;
	return LabelList.pElement[ExampleIndex];
}

double CLabeledDataSetForClassification_ByGroupedFeature::GetExampleInitialWeight(int ExampleIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return 0.0;
	return InitialWeightList.pElement[ExampleIndex];
}

double CLabeledDataSetForClassification_ByGroupedFeature::GetExampleFeature(int ExampleIndex, int FeatureIndex)
{
	if (ExampleIndex<0 || ExampleIndex>=ExampleAmount) return 0.0;
	if (FeatureIndex<0 || FeatureIndex>=FeatureDim) return 0.0;

	if (!IsFeatureValid(FeatureIndex))
	{
		int i;
		for ( i = 0; i < grp_pool_size; ++i )
			if (cache2grp.pElement[i] < 0)
				break;
		if (i >= grp_pool_size) i = CMyRandom::NextInt(grp_pool_size);

		LoadGroup(fea2grp.pElement[FeatureIndex], i);
	}
	
	return fv_headers.pElement[FeatureIndex][ExampleIndex];
}

void CLabeledDataSetForClassification_ByGroupedFeature::InitValidFeatures()
{
	int i,j;
	for ( i = 0; i < grp_pool_size; ++i )
	{
		if (cache2grp.pElement[i] >= 0) continue;
		j = RandomlySelectUncachedGroup();
		if (j >= 0) LoadGroup(j, i);
	}
}

int CLabeledDataSetForClassification_ByGroupedFeature::GetValidFeatureAmount()
{
	return in_cache_fea_amount;
}

int CLabeledDataSetForClassification_ByGroupedFeature::RandomlySelectUncachedGroup()
{
	int j, t;
	t = 0; for ( j = 0; j < grp_amount; ++j ) if (grp2cache.pElement[j] < 0) ++t;
	t = CMyRandom::NextInt(t); for ( j = 0; j < grp_amount; ++j ) if (grp2cache.pElement[j] < 0)
	{
		if (t==0) return j;
		--t;
	}
	return -1;
}

bool CLabeledDataSetForClassification_ByGroupedFeature::IsFeatureValid(int FeatureIndex)
{
	if (FeatureIndex<0 || FeatureIndex>=FeatureDim) return false;
	return (fv_headers.pElement[FeatureIndex] != NULL);
}

void CLabeledDataSetForClassification_ByGroupedFeature::UpdateValidFeatures()
{
	int i,j;

	j = RandomlySelectUncachedGroup();
	if (j < 0) return;

	for ( i = 0; i < grp_pool_size; ++i )
		if (cache2grp.pElement[i] < 0)
			break;
	if (i >= grp_pool_size) i = CMyRandom::NextInt(grp_pool_size);

	LoadGroup(j, i);
}

void CLabeledDataSetForClassification_ByGroupedFeature::LoadGroup(int GroupIndex, int CachePosition)
{
	int i, ii, j, t;

	if (CachePosition<0 || CachePosition>=grp_pool_size) return;

	j = cache2grp.pElement[CachePosition];
	if (j >= 0)
	{
		ii = grp_length.pElement[j];
		in_cache_fea_amount -= ii;
		i = grp_headers.pElement[j];
		ii += i;
		for ( ; i < ii; ++i )
			fv_headers.pElement[fea_index.pElement[i]] = NULL;
		grp2cache.pElement[j] = -1;
		cache2grp.pElement[CachePosition] = -1;
	}

	if (GroupIndex<0 || GroupIndex>=grp_amount) return;

	j = GroupIndex;
	ii = grp_length.pElement[j];
	in_cache_fea_amount += ii;
	i = grp_headers.pElement[j];

	double* pd = fv_data.pElement + (size_t)ExampleAmount * (size_t)per_grp_size * (size_t)CachePosition;
	_fseeki64(FIN, _BaseOffset + _DataBlockSize * i, SEEK_SET);
	CDataStructureTools::my_fread(FIN, pd, ((size_t)ExampleAmount) * (size_t)ii);

	ii += i;
	for ( ; i < ii; ++i )
	{
		t = fea_index.pElement[i];
		fv_headers.pElement[t] = pd;
		pd += ExampleAmount;
	}
	grp2cache.pElement[j] = CachePosition;
	cache2grp.pElement[CachePosition] = j;
}

